# -*- coding: utf-8 -*-

# Author: 一根鱼骨棒
# Date: 2020-09-17 14:52:15
# LastEditTime: 2023-03-07 09:17:34
# LastEditors: 一根鱼骨棒
# Description: 本开源代码使用GPL 3.0协议
# Software: VScode
# Copyright 2020 迷舍

import json
import os
import re
import math
import time
import random
from multiprocessing import Process
from subprocess import call
from cv2 import cv2
import jieba.analyse
import requests
from lxml import etree
import datetime
import html
from git.repo import Repo
from git import Git
from git.repo.fun import is_git_dir
# 今天日期
today = datetime.date.today()
# 昨天时间
yesterday = today - datetime.timedelta(days=1)
# 时间
year = time.strftime("%Y", time.localtime())
month = time.strftime("%m", time.localtime())
day = time.strftime("%d", time.localtime())
week = time.strftime("%W", time.localtime())
hours = time.strftime("%H", time.localtime())
date = str(year)+str(month)+str(day)

# 常量定义
DEBUG = False
# 关键字分隔符
KEYWORD_SEPARATOR = "|"
# 视频下载开关
DOWNLOAD_VIDEO = True
# 图片下载开关
DOWNLOAD_PIC = True
# IDM下载器路径
IDM = r'D:\Program Files (x86)\IDM6.30\IDMan.exe'
# 目标URL，采集后上传的图片地址，直接替换到正文中
TARGET_URL = 'http://11.176.40.52/qjw/asset/'+str(year)+str(month)+str(day)+"/"
# 模型ID
MODEL_ID = '1'
user_agent = [
      "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; AcooBrowser; .NET CLR 1.1.4322; .NET CLR 2.0.50727)",
    "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.0; Acoo Browser; SLCC1; .NET CLR 2.0.50727; Media Center PC 5.0; .NET CLR 3.0.04506)",
    "Mozilla/4.0 (compatible; MSIE 7.0; AOL 9.5; AOLBuild 4337.35; Windows NT 5.1; .NET CLR 1.1.4322; .NET CLR 2.0.50727)",
    "Mozilla/5.0 (Windows; U; MSIE 9.0; Windows NT 9.0; en-US)",
    "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Win64; x64; Trident/5.0; .NET CLR 3.5.30729; .NET CLR 3.0.30729; .NET CLR 2.0.50727; Media Center PC 6.0)",
    "Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 6.0; Trident/4.0; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; .NET CLR 1.0.3705; .NET CLR 1.1.4322)",
    "Mozilla/4.0 (compatible; MSIE 7.0b; Windows NT 5.2; .NET CLR 1.1.4322; .NET CLR 2.0.50727; InfoPath.2; .NET CLR 3.0.04506.30)",
    "Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN) AppleWebKit/523.15 (KHTML, like Gecko, Safari/419.3) Arora/0.3 (Change: 287 c9dfb30)",
    "Mozilla/5.0 (X11; U; Linux; en-US) AppleWebKit/527+ (KHTML, like Gecko, Safari/419.3) Arora/0.6",
    "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8.1.2pre) Gecko/20070215 K-Ninja/2.1.1",
    "Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN; rv:1.9) Gecko/20080705 Firefox/3.0 Kapiko/3.0",
    "Mozilla/5.0 (X11; Linux i686; U;) Gecko/20070322 Kazehakase/0.4.5",
    "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.0.8) Gecko Fedora/1.9.0.8-1.fc10 Kazehakase/0.5.6",
    "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.56 Safari/535.11",
    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_3) AppleWebKit/535.20 (KHTML, like Gecko) Chrome/19.0.1036.7 Safari/535.20",
    "Opera/9.80 (Macintosh; Intel Mac OS X 10.6.8; U; fr) Presto/2.9.168 Version/11.52",
    "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.11 (KHTML, like Gecko) Chrome/20.0.1132.11 TaoBrowser/2.0 Safari/536.11",
    "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/21.0.1180.71 Safari/537.1 LBBROWSER",
    "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C; .NET4.0E; LBBROWSER)",
    "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; QQDownload 732; .NET4.0C; .NET4.0E; LBBROWSER)",
    "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.84 Safari/535.11 LBBROWSER",
    "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C; .NET4.0E)",
    "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C; .NET4.0E; QQBrowser/7.0.3698.400)",
    "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; QQDownload 732; .NET4.0C; .NET4.0E)",
    "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Trident/4.0; SV1; QQDownload 732; .NET4.0C; .NET4.0E; 360SE)",
    "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; QQDownload 732; .NET4.0C; .NET4.0E)",
    "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C; .NET4.0E)",
    "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/21.0.1180.89 Safari/537.1",
    "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/21.0.1180.89 Safari/537.1",
    "Mozilla/5.0 (iPad; U; CPU OS 4_2_1 like Mac OS X; zh-cn) AppleWebKit/533.17.9 (KHTML, like Gecko) Version/5.0.2 Mobile/8C148 Safari/6533.18.5",
    "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:2.0b13pre) Gecko/20110307 Firefox/4.0b13pre",
    "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:16.0) Gecko/20100101 Firefox/16.0",
    "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.64 Safari/537.11",
    "Mozilla/5.0 (X11; U; Linux x86_64; zh-CN; rv:1.9.2.10) Gecko/20100922 Ubuntu/10.10 (maverick) Firefox/3.6.10"
]
headers = {
    'User-Agent':random.choice(user_agent)
}
# 下载的目录
remote = "https://gitee.com/truelyluser/news_log.git"
download_path = os.getcwd()+'\\news\\'+date+hours+"spider\\"
log_path = os.getcwd()+'\\news\\logs\\'
log_name = year+week+"added.txt"
download_asset_path = download_path+'asset/'

if not (os.path.exists(download_asset_path)):
    os.makedirs(download_asset_path)
if not (os.path.exists(log_path)):
    os.makedirs(log_path)
if not (os.path.isfile(log_path+'count.txt')):
    with open(log_path+'count.txt','w') as f:
        f.write("")

def cleanChineseCharacter(text):
    '''
    处理特殊的中文符号,将其全部替换为'-' 否则在保存时Windows无法将有的中文符号作为路径\n
    @param text: 固定参数，传入需要转换的字符串\n
    @return:返回生成的字符串\n
    '''
    text = str(text)
    chars = ["/", r"?", r"!", r"|", "\"", "'", "·", "。", "？", "！", "，", "、", "；",
             "：", "‘", "’", "“", "”", "（", "）", "…", "–", "．", "《", "》"]
    new_text = ""
    lens = len(text)
    for i in range(lens):
        if text[i] not in chars:
            new_text += text[i]
        else:
            new_text += "_"
    return new_text

# Common


def idmDownloader(idm, url, fpath, filename):
    '''
    封装的IDM下载器\n
    @param idm: 传入idm路径\n
    @param url: 传入下载地址\n
    @param fpath: 传入保存路径\n
    @param filename: 传入文件名\n
    @param video_count: 传入计数避免同名\n
    @return 无返回值
    '''
    if os.path.isfile(fpath+filename):
        print("文件已存在")
        return
    print("IDM 下载："+url)
    try:
        call([idm, '/d', url, '/p', fpath,
              '/f', filename, '/q ', '/n'])
    except:
        print("调用下载器失败，IDM程序当前路径：{}".format(idm))

# 判断是否含有中文Common


def hasChinese(string):
    '''
    判断字符串内是否有中文字符
    @param string: 固定参数，传入保存路径
    @return {Bool}
    '''
    RE = re.compile(u'[\u4e00-\u9fa5]', re.UNICODE)
    match = re.search(RE, string)
    if match is None:
        return False
    else:
        return True

# 获取etree编码的html Common


def getHtml(url, headers=""):
    '''
    获取html树\n
    @param url:链接\n
    @return {element}：html检索树\n
    '''
    # 获取要采集的文章url列表
    r = requests.get(url, headers=headers)
    r.close()
    # print(r.content.decode('utf-8'))
    html = etree.HTML(r.content)
    return html


def getJson(url, headers=""):
    '''
    获取json格式的列表数据
    '''
    r = requests.get(url, headers=headers)
    try:
        result = json.loads(r.text)
    except:
        print("json解析失败")
        result = json.loads(
            "{'code':'0','data':'null','message':'json decode failed'}")
    return result
# 设置摘要Common


def setSummary(text, count=100, suffix=u'...'):
    """

    """
    assert(isinstance(text, str))
    summary = re.sub(r'<.*?>', u'', text)  # key difference: use regex
    summary = u''.join(summary.split())[0:count]
    return u'{0}{1}'.format(summary, suffix)

# 设置关键字Common


def setKeywords(content, separator="|"):
    '''
    设置关键字\n
    @param {string} content 内容\n
    @return {string} 关键字字符串\n
    '''
    keywords_textrank = jieba.analyse.textrank(content, topK=3)
    # 生成的列表转换为字符串
    result = (separator.join(keywords_textrank))
    return result


def saveFailInfo(info):
    '''
    保存错误信息
    @param {string} info 错误信息
    '''
    return
    with open(download_path+'failed.txt', 'a+') as target:
        target.write(info+'\r\n')


def resizePicture(img_list,):
    '''
    如果图片尺寸不符合要求，则调整图片\n
    @param img_list {list}:图片列表\n

    '''
    for each in img_list:
        each = each.split('/')[-1]
        img = cv2.imread(download_asset_path+each.split('/')[-1])
        try:
            width, height = img.shape[1], img.shape[0]
        except:
            print("未读取到图片")
            continue
        scale = (height/width)
        if int(width) > 900:
            img = cv2.resize(img, (700, int(700*scale)))
        if int(width) < 500:
            img = cv2.resize(img, (500, int(500*scale)))
        cv2.imwrite(download_asset_path+each, img)
    # print("尺寸调整完成")
    return
# 设置缩略图


def setThumb(img_list):
    '''
    设置缩略图,新华军事和网评的缩略图在列表页，如果图片尺寸不符合要求，则重新选图片\n
    @param content {list}:内容\n
    @return {url} thumb \n
    '''
    thumb = ""
    for each in img_list:
        print(each)
        if ".gif" in each:
            continue
        each = each.split('/')[-1]
        img = cv2.imread(download_asset_path+each)
        try:
            width, height = img.shape[1], img.shape[0]
        except:
            print("设置大小失败")
            return thumb
        if float('%.1f' % (height/width)) > 2 or int(width) < 500:
            print("图片大小不恰当")
            continue
        else:
            thumb = TARGET_URL+each
            print("设置缩略图成功")
            # print("thumb"+thumb)
            break
    return thumb
# 常规分隔模式


def segTailSplit(str1, sep=r"</p>"):
    '''
    保留分割符号，置于句尾，比如标点符号,分隔符可为多样的正则表达式
    '''
    result = re.split(sep, str1)
    seg_word = re.findall(sep, str1)
    seg_word.extend(" ")  # 末尾插入一个空字符串，以保持长度和切割成分相同
    result = [x+y for x, y in zip(result, seg_word)]  # 顺序可根据需求调换
    return result

# git 相关命令

class GitRepository(object):
    """
    git仓库管理
    """

    def __init__(self, local_path, repo_url, branch='master'):
        self.local_path = local_path
        self.repo_url = repo_url
        self.repo = None
        self.initial(repo_url, branch)

    def initial(self, repo_url, branch):
        """
        初始化git仓库
        :param repo_url:
        :param branch:
        :return:
        """
        if not os.path.exists(self.local_path):
            os.makedirs(self.local_path)

        git_local_path = os.path.join(self.local_path, '.git')
        if not is_git_dir(git_local_path):
            self.repo = Repo.clone_from(
                repo_url, to_path=self.local_path, branch=branch)
        else:
            self.repo = Repo(self.local_path)

    def pull(self):
        """
        从线上拉最新代码
        :return:
        """
        self.repo.git.pull()

    def branches(self):
        """
        获取所有分支
        :return:
        """
        branches = self.repo.remote().refs
        return [item.remote_head for item in branches if item.remote_head not in ['HEAD', ]]

    def commits(self):
        """
        获取所有提交记录
        :return:
        """
        commit_log = self.repo.git.log('--pretty={"commit":"%h","author":"%an","summary":"%s","date":"%cd"}',
                                       max_count=50,
                                       date='format:%Y-%m-%d %H:%M')
        log_list = commit_log.split("\n")
        return [eval(item) for item in log_list]

    def change_to_commit(self, branch, commit):
        """
        切换commit
        :param branch:
        :param commit:
        :return:
        """
        self.change_to_branch(branch=branch)
        self.repo.git.reset('--hard', commit)




